<!DOCTYPE html><html class="theme-next gemini use-motion" lang="zh-CN"><head><meta name="generator" content="Hexo 3.9.0"><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=2"><meta name="theme-color" content="#222"><link rel="apple-touch-icon" sizes="180x180" href="/favicon.ico?v=6.7.0"><link rel="icon" type="image/png" sizes="32x32" href="/favicon.ico?v=6.7.0"><link rel="icon" type="image/png" sizes="16x16" href="/favicon.ico?v=6.7.0"><script id="hexo.configurations">var NexT=window.NexT||{},CONFIG={root:"/",scheme:"Gemini",version:"6.7.0",sidebar:{position:"left",display:"post",offset:12,b2t:!1,scrollpercent:!1,onmobile:!1},fancybox:!0,fastclick:!0,lazyload:!1,tabs:!0,motion:{enable:!0,async:!1,transition:{post_block:"fadeIn",post_header:"slideDownIn",post_body:"slideDownIn",coll_header:"slideLeftIn",sidebar:"slideUpIn"}},algolia:{applicationID:"",apiKey:"",indexName:"",hits:{per_page:10},labels:{input_placeholder:"Search for Posts",hits_empty:"We didn't find any results for the search: ${query}",hits_stats:"${hits} results found in ${time} ms"}}}</script><meta name="description" content="Envoy 中包含了一个精心设计的统计子系统，本文将从源码角度分析该子系统的设计目标及特点。"><meta name="keywords" content="envoy,service mesh,源码分析"><meta property="og:type" content="article"><meta property="og:title" content="Envoy 统计数据子系统源码分析"><meta property="og:url" content="http://moeyui.cn/开发/7c949937.html"><meta property="og:site_name" content="不是很懂"><meta property="og:description" content="Envoy 中包含了一个精心设计的统计子系统，本文将从源码角度分析该子系统的设计目标及特点。"><meta property="og:locale" content="zh-CN"><meta property="og:image" content="https://s1.ax1x.com/2018/09/23/iulKMV.png"><meta property="og:image" content="https://s1.ax1x.com/2018/09/23/iul8IJ.png"><meta property="og:updated_time" content="2018-09-24T16:15:12.324Z"><meta name="twitter:card" content="summary"><meta name="twitter:title" content="Envoy 统计数据子系统源码分析"><meta name="twitter:description" content="Envoy 中包含了一个精心设计的统计子系统，本文将从源码角度分析该子系统的设计目标及特点。"><meta name="twitter:image" content="https://s1.ax1x.com/2018/09/23/iulKMV.png"><link rel="alternate" href="/atom.xml" title="不是很懂" type="application/atom+xml"><link rel="canonical" href="http://moeyui.cn/开发/7c949937.html"><script id="page.configurations">CONFIG.page={sidebar:""}</script><title>Envoy 统计数据子系统源码分析 | 不是很懂</title><script async="" src="//www.googletagmanager.com/gtag/js?id=UA-75483578-1"></script><script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","UA-75483578-1")</script><noscript><style>.sidebar-inner,.use-motion .brand,.use-motion .collection-title,.use-motion .comments,.use-motion .menu-item,.use-motion .motion-element,.use-motion .pagination,.use-motion .post-block,.use-motion .post-body,.use-motion .post-header{opacity:initial}.use-motion .logo,.use-motion .site-subtitle,.use-motion .site-title{opacity:initial;top:initial}.use-motion .logo-line-before i{left:initial}.use-motion .logo-line-after i{right:initial}</style></noscript><script>function loadCss(e){var t=document,s=t.head,n=t.createElement("link");n.rel="stylesheet",n.href=e,function e(s){if(t.body)return s();setTimeout(function(){e(s)})}(function(){s.appendChild(n)})}loadCss("/style.css"),loadCss("https://cdn.staticfile.org/font-awesome/4.6.2/css/font-awesome.min.css")</script><noscript><link rel="stylesheet" href="/style.css"><link rel="stylesheet" href="https://cdn.staticfile.org/font-awesome/4.6.2/css/font-awesome.min.css"></noscript></head><body itemscope="" itemtype="http://schema.org/WebPage" lang="zh-CN"><div class="container sidebar-position-left page-post-detail"><div class="headband"></div><header id="header" class="header" itemscope="" itemtype="http://schema.org/WPHeader"><div class="header-inner"><div class="site-brand-wrapper"><div class="site-meta"><div class="custom-logo-site-title"><a href="/" class="brand" rel="start"><span class="logo-line-before"><i></i></span> <span class="site-title">不是很懂</span><span class="logo-line-after"><i></i></span></a></div><h1 class="site-subtitle" itemprop="description">不是很懂你们程序员</h1></div><div class="site-nav-toggle"> <button aria-label="切换导航栏"><span class="btn-bar"></span><span class="btn-bar"></span><span class="btn-bar"></span></button></div></div><nav class="site-nav"><ul id="menu" class="menu"><li class="menu-item menu-item-home"><a href="/" rel="section"><i class="menu-item-icon fa fa-fw fa-home"></i><br>首页</a></li><li class="menu-item menu-item-about"><a href="/about/" rel="section"><i class="menu-item-icon fa fa-fw fa-user"></i><br>关于</a></li><li class="menu-item menu-item-tags"><a href="/tags/" rel="section"><i class="menu-item-icon fa fa-fw fa-tags"></i><br>标签</a></li><li class="menu-item menu-item-categories"><a href="/categories/" rel="section"><i class="menu-item-icon fa fa-fw fa-th"></i><br>分类</a></li><li class="menu-item menu-item-archives"><a href="/archives/" rel="section"><i class="menu-item-icon fa fa-fw fa-archive"></i><br>归档</a></li></ul></nav></div></header> <a href="https://github.com/moeyui1" class="github-corner" title="Follow me on GitHub" aria-label="Follow me on GitHub" rel="noopener" target="_blank"><svg width="80" height="80" viewBox="0 0 250 250" style="fill:#222;color:#fff;position:absolute;top:0;border:0;right:0" aria-hidden="true"><path d="M0 0 115 115 130 115 142 142 250 250 250 0Z"></path><path d="M128.3 109C113.8 99.7 119 89.6 119 89.6 122 82.7 120.5 78.6 120.5 78.6 119.2 72 123.4 76.3 123.4 76.3 127.3 80.9 125.5 87.3 125.5 87.3 122.9 97.6 130.6 101.9 134.4 103.2" fill="currentColor" style="transform-origin:130px 106px" class="octo-arm"></path><path d="M115 115C114.9 115.1 118.7 116.5 119.8 115.4L133.7 101.6C136.9 99.2 139.9 98.4 142.2 98.6 133.8 88 127.5 74.4 143.8 58 148.5 53.4 154 51.2 159.7 51 160.3 49.4 163.2 43.6 171.4 40.1 171.4 40.1 176.1 42.5 178.8 56.2 183.1 58.6 187.2 61.8 190.9 65.4 194.5 69 197.7 73.2 200.1 77.6 213.8 80.2 216.3 84.9 216.3 84.9 212.7 93.1 206.9 96 205.4 96.6 205.1 102.4 203 107.8 198.3 112.5 181.9 128.9 168.3 122.5 157.7 114.1 157.9 116.9 156.7 120.9 152.7 124.9L141 136.5C139.8 137.7 141.6 141.9 141.8 141.8Z" fill="currentColor" class="octo-body"></path></svg></a><main id="main" class="main"><div class="main-inner"><div class="content-wrap"><div id="content" class="content"><div id="posts" class="posts-expand"><article class="post post-type-normal" itemscope="" itemtype="http://schema.org/Article"><div class="post-block"><link itemprop="mainEntityOfPage" href="http://moeyui.cn/开发/7c949937.html"><span hidden="" itemprop="author" itemscope="" itemtype="http://schema.org/Person"><meta itemprop="name" content="moeyui"><meta itemprop="description" content="moeyui | moeyui's blog"><meta itemprop="image" content="/images/blogAvatar.jpg"></span><span hidden="" itemprop="publisher" itemscope="" itemtype="http://schema.org/Organization"><meta itemprop="name" content="不是很懂"></span><header class="post-header"><h2 class="post-title" itemprop="name headline">Envoy 统计数据子系统源码分析</h2><div class="post-meta"><span class="post-time"><span class="post-meta-item-icon"><i class="fa fa-calendar-o"></i></span> <span class="post-meta-item-text">发表于</span> <time title="创建时间：2018-09-25 00:10:42 / 修改时间：00:15:12" itemprop="dateCreated datePublished" datetime="2018-09-25T00:10:42+08:00">2018-09-25</time></span> <span class="post-category"><span class="post-meta-divider">|</span><span class="post-meta-item-icon"><i class="fa fa-folder-o"></i></span> <span class="post-meta-item-text">分类于</span> <span itemprop="about" itemscope="" itemtype="http://schema.org/Thing"><a href="/categories/开发/" itemprop="url" rel="index"><span itemprop="name">开发</span></a></span></span> <span class="post-comments-count"><span class="post-meta-divider">|</span><span class="post-meta-item-icon"><i class="fa fa-comment-o"></i></span> <a href="/开发/7c949937.html#comments" itemprop="discussionUrl"><span class="post-meta-item-text">评论数：</span><span class="post-comments-count valine-comment-count" data-xid="/开发/7c949937.html" itemprop="commentCount"></span></a></span> <span id="/开发/7c949937.html" class="leancloud_visitors" data-flag-title="Envoy 统计数据子系统源码分析"><span class="post-meta-divider">|</span><span class="post-meta-item-icon"><i class="fa fa-eye"></i></span> <span class="post-meta-item-text">阅读次数：</span><span class="leancloud-visitors-count"></span></span><div class="post-description">Envoy 中包含了一个精心设计的统计子系统，本文将从源码角度分析该子系统的设计目标及特点。</div></div></header><div class="post-body" itemprop="articleBody"><p>关于 Envoy 监控的相关概念可以参考<a href="https://blog.envoyproxy.io/envoy-stats-b65c7f363342" target="_blank" rel="noopener">官方博客</a>。不过这篇博客有点旧，一些概念应以最新的代码为准。下面简单介绍一些基本概念。</p><h1 id="设计初衷"><a href="#设计初衷" class="headerlink" title="设计初衷"></a>设计初衷</h1><p>关于为何 envoy 要设计一套复杂的统计子系统，这里直接引用官方的 blog 内容：</p><ul><li><p>在任意数量工作线程下保持大致线性的吞吐量。换一种说法，在稳定状态下使用统计时，应该是没有跨线程抢占的。</p><blockquote><p>Roughly linear throughput that scales with any number of worker threads. Said another way: at steady state there should be zero cross-thread contention when using stats.</p></blockquote></li><li><p>当使用热重启时，统计应该是逻辑上一致的。这意味着即使有两个Envoy线程在运行，所有的 counters、gauges 和 histograms 在逻辑上当作是一个进程内时，都应该是一致的。</p><blockquote><p>When using hot restart, stats should be logically consistent. This means that even when there are two Envoy processes running, all of the counters, gauges, and histograms should be consistent when logically considered as a single process. (See the <a href="https://medium.com/@mattklein123/envoy-hot-restart-1d16b14555b5" target="_blank" rel="noopener">hot restart post</a> for more information on this).</p></blockquote></li><li><p>统计信息应包含在域（scopes）中，并以组为单位释放。域是包含共同前缀统计信息的逻辑分组。例如：<code>http.admin.*</code>。这一点由于Envoy的动态性而十分重要。Envoy支持多种管理APIs诸如监听器发现服务（LDS）和集群发现服务（CDS）APIs。为了不占满内存，Envoy 需要清理已经用不到的统计数据。</p><blockquote><p>Stats should be contained within <em>scopes</em> and freed as a group. A scope is a logical grouping of stats with a common prefix. For example: http.admin.*. This is important because of the dynamic nature of Envoy. Envoy supports various <a href="https://lyft.github.io/envoy/docs/intro/arch_overview/dynamic_configuration.html" target="_blank" rel="noopener">management APIs</a> such as the Listener Discovery Service (LDS) and Cluster Discovery Service (CDS) APIs. In order to not run out of memory Envoy needs to clean up stats that are no longer used.</p></blockquote></li><li><p>统计域应该能覆盖和正确地进行引用计数.举个例子，域A使用了 <code>foo.bar.baz</code> ，域B也用了 <code>foo.bar.baz</code>，那么底层的 <code>foo.bar.baz</code> 应该引用计数为2。这对于热重启（某个时间段两个进程都会写同一个统计数据）和动态管理接口（某个时间段一个已更新的 listener 或 cluster 会与未更新的 listener 或 cluster 引用同一个统计数据）是必要的。</p><blockquote><p>Stat scopes should be capable of being <em>overlapped</em> and properly reference counted. This means that if scope A uses a stat called <em>foo.bar.baz</em> and scope B also uses <em>foo.bar.baz</em>, the underlying <em>foo.bar.baz</em> stat should have a reference count of two. This is required both for hot restart (both processes will write to the same stats for some period of time) as well as for the dynamic management APIs (for some period of time an updated listener or cluster will reference the same stats as the old listener or cluster).</p></blockquote></li><li><p>对高低频统计数据的分别优化。</p><blockquote><p>The stat subsystem should perform well for stats that may not be known until data plane processing starts. Many stats are essentially “fixed” and can be created when the configuration is loaded or the dynamic APIs reconfigure the data plane (e.g., cluster.foo.upstream_rq_5xx). These are both low frequency events. Other stats, such as detailed HTTP response code metrics (e.g., cluster.foo.upstream_rq_503), are not known until data starts flowing. Using “dynamic” stats is never going to be as fast as using “fixed” stats but performance should still be adequate even when processing 10’s of thousands of requests per second per core.</p></blockquote></li></ul><h1 id="Overview"><a href="#Overview" class="headerlink" title="Overview"></a>Overview</h1><p>首先介绍开源界监控的一般解决方案，以 dogstatd 为例：</p><p><a href="https://imgchr.com/i/iulKMV" target="_blank" rel="noopener"><img src="https://s1.ax1x.com/2018/09/23/iulKMV.png" alt="iulKMV.png"></a></p><p>其中 DogStatsd 单独部署，负责聚合、收集统计数据，为 Envoy 中 Sink 一角。下文主要讨论处于 DogStatsd 下游的应用代码中的实现，探究 Envoy 如何处理统计数据，并将它们发给 Sink 的。</p><p>Envoy 目前支持3种监控指标类型：</p><ul><li>Counters（计数器）：无符号整数只增不减，如总请求数；</li><li>Gauges（量表）：无符号整数可增可减，如目前有效请求数；</li><li>Histograms（直方图）：作为值流的一部分的无符号整数，然后由收集器进行汇总以最终生成汇总百分点值。 例如，上游请求时间。</li></ul><p>在内部，计数器和计量器被分批并定期冲洗以提高性能。直方图会在收到时写入。 注意：以前称为定时器的内容已成为直方图，因为这两种表示法之间的唯一区别就是单位（秒和毫秒）。</p><h1 id="结构"><a href="#结构" class="headerlink" title="结构"></a>结构</h1><p><a href="https://imgchr.com/i/iul8IJ" target="_blank" rel="noopener"><img src="https://s1.ax1x.com/2018/09/23/iul8IJ.png" alt="iul8IJ.png"></a></p><p>上图是统计系统的大体结构。</p><h2 id="Store"><a href="#Store" class="headerlink" title="Store"></a>Store</h2><blockquote><p>The stat store is a singleton within Envoy and provides a simple interface by which the rest of the code can obtain handles to scopes, counters, gauges, and histograms. Calling code is responsible for maintaining ownership semantics of any created scopes. When a scope is destroyed, all of the contained stats have their reference count decreased by one. If any stats reach a reference count of zero they will be freed.</p></blockquote><p>由于 Store 的实现比较贴近底层，主要考虑内存管理、锁方面的问题，故本文不做深入分析。</p><figure class="highlight c++"><table><tbody><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment"> * A store for all known counters, gauges, and timers.</span></span><br><span class="line"><span class="comment"> */</span></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Store</span> :</span> <span class="keyword">public</span> Scope {</span><br><span class="line"><span class="keyword">public</span>:</span><br><span class="line">  <span class="comment">/**</span></span><br><span class="line"><span class="comment">   * @return a list of all known counters.</span></span><br><span class="line"><span class="comment">   */</span></span><br><span class="line">  <span class="keyword">virtual</span> <span class="built_in">std</span>::<span class="built_in">vector</span>&lt;CounterSharedPtr&gt; counters() <span class="keyword">const</span> PURE;</span><br><span class="line"></span><br><span class="line">  <span class="comment">/**</span></span><br><span class="line"><span class="comment">   * @return a list of all known gauges.</span></span><br><span class="line"><span class="comment">   */</span></span><br><span class="line">  <span class="keyword">virtual</span> <span class="built_in">std</span>::<span class="built_in">vector</span>&lt;GaugeSharedPtr&gt; gauges() <span class="keyword">const</span> PURE;</span><br><span class="line"></span><br><span class="line">  <span class="comment">/**</span></span><br><span class="line"><span class="comment">   * @return a list of all known histograms.</span></span><br><span class="line"><span class="comment">   */</span></span><br><span class="line">  <span class="keyword">virtual</span> <span class="built_in">std</span>::<span class="built_in">vector</span>&lt;ParentHistogramSharedPtr&gt; histograms() <span class="keyword">const</span> PURE;</span><br><span class="line">};</span><br></pre></td></tr></tbody></table></figure><h2 id="Stats"><a href="#Stats" class="headerlink" title="Stats"></a>Stats</h2><p>As described previously, stats include counters, gauges, and histograms. From an end user perspective, these interfaces are very simple to use. For example, counters and gauges include an inc() and dec() method while only gauges include a set() method. Any underlying storage complexity is hidden from the programmer.</p><h2 id="Flusher"><a href="#Flusher" class="headerlink" title="Flusher"></a>Flusher</h2><p>To achieve high performance, Envoy internally buffers all stat changes using atomic CPU instructions. At a configurable interval all of the counters and gauges are flushed to the sinks. Note that in the current architecture <em>histogram values are sent directly to the sinks</em>. This will be described in more detail below. The flusher runs on the main thread.</p><h2 id="Sinks"><a href="#Sinks" class="headerlink" title="Sinks"></a>Sinks</h2><p>A stat sink is an interface that takes generic stat data and translates it into a backend-specific wire format. All sinks utilize TLS so that there is no contention when flushing output. In practice however, only the main thread currently flushes counters and gauges. All threads flush histograms.</p><p>Currently, Envoy only supports the TCP and UDP <a href="https://github.com/b/statsd_spec" target="_blank" rel="noopener">statsd</a> protocol. statsd is an incredibly simple but very widely supported transport format. In the future it is very likely that other native stat sinks will be implemented, such as <a href="https://prometheus.io/" target="_blank" rel="noopener">Prometheus</a>, <a href="https://www.wavefront.com/" target="_blank" rel="noopener">Wavefront</a>, and <a href="https://www.influxdata.com/" target="_blank" rel="noopener">InfluxDB</a>. Note also that Envoy does not currently support <em>dimensional</em> or <em>tagged</em> stats. This will be discussed further in the future work section below.</p><p>The built-in stats sinks are:</p><ul><li><a href="https://www.envoyproxy.io/docs/envoy/latest/api-v2/config/metrics/v2/stats.proto#envoy-api-msg-config-metrics-v2-statsdsink" target="_blank" rel="noopener">envoy.statsd</a></li><li><a href="https://www.envoyproxy.io/docs/envoy/latest/api-v2/config/metrics/v2/stats.proto#envoy-api-msg-config-metrics-v2-dogstatsdsink" target="_blank" rel="noopener">envoy.dog_statsd</a></li><li><a href="https://www.envoyproxy.io/docs/envoy/latest/api-v2/config/metrics/v2/metrics_service.proto#envoy-api-msg-config-metrics-v2-metricsserviceconfig" target="_blank" rel="noopener">envoy.metrics_service</a></li><li><a href="https://www.envoyproxy.io/docs/envoy/latest/api-v2/config/metrics/v2/stats.proto#envoy-api-msg-config-metrics-v2-hystrixsink" target="_blank" rel="noopener">envoy.stat_sinks.hystrix</a></li></ul><h2 id="Admin"><a href="#Admin" class="headerlink" title="Admin"></a>Admin</h2><p>From an operations perspective, it is incredibly useful to be able to get onto a node and dump the current stats in realtime. Envoy enables this via the /stats <a href="https://lyft.github.io/envoy/docs/operations/admin.html#get--stats" target="_blank" rel="noopener">admin endpoint</a>. The admin endpoint looks directly into the store to load all of the counters and gauges and print them. This endpoint does not currently output any histogram data. This again is due to the fact that in the current implementation histogram values are written directly to the sinks so the store does not know about them.</p><h1 id="源码追踪"><a href="#源码追踪" class="headerlink" title="源码追踪"></a>源码追踪</h1><h2 id="执行流程"><a href="#执行流程" class="headerlink" title="执行流程"></a>执行流程</h2><h3 id="数据推送"><a href="#数据推送" class="headerlink" title="数据推送"></a>数据推送</h3><p>流程复杂又繁琐，整理了一下<a href="https://oeoiy7i1f.qnssl.com/envoy_stats.xmind" target="_blank" rel="noopener">思维导图</a>。粗略地讲，就是定时调用 <code>Stats::Sink</code> 的 flush 方法，将 <code>Stats::Source</code> 中的统计数据推给 sink。</p><h3 id="埋点"><a href="#埋点" class="headerlink" title="埋点"></a>埋点</h3><p>envoy 这一套抽象得很好，在埋点的时候使用的实际是 <code>Stats::Store</code>，将数据存到本地的 cache 中，再通过上文的“数据推送”定时推到后端的 sink。下面以 <code>envoy/source/extensions/filters/network/ratelimit/ratelimit.cc</code> 为例追踪下埋点的源码实现，比较复杂繁琐，可以选择性跳过，简单来说就是埋点处对 store 层数据进行修改。</p><h4 id="Network-FilterStatus-Filter-onNewConnection"><a href="#Network-FilterStatus-Filter-onNewConnection" class="headerlink" title="Network::FilterStatus Filter::onNewConnection()"></a>Network::FilterStatus Filter::onNewConnection()</h4><figure class="highlight c++"><table><tbody><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">//...</span></span><br><span class="line">  <span class="keyword">if</span> (status_ == Status::NotStarted) {</span><br><span class="line">    status_ = Status::Calling;</span><br><span class="line">    config_-&gt;stats().active_.inc();</span><br><span class="line">    config_-&gt;stats().total_.inc();</span><br><span class="line">    calling_limit_ = <span class="literal">true</span>;</span><br><span class="line">    client_-&gt;limit(*<span class="keyword">this</span>, config_-&gt;domain(), config_-&gt;descriptors(), Tracing::NullSpan::instance());</span><br><span class="line">    calling_limit_ = <span class="literal">false</span>;</span><br><span class="line">  }</span><br><span class="line"><span class="comment">//...</span></span><br></pre></td></tr></tbody></table></figure><p>该方法中有一处埋点，调用了 <code>config_</code> 变量的 <code>stats()</code> 方法。<code>config_</code> 实际上是一个 <code>RateLimitFilter::Config</code> 类型的实例。<code>stats()</code> 返回 <code>config_</code> 持有的一个 <code>InstanceStats</code> 类型的引用。下面看 <code>InstanceStats</code> 的代码。</p><h4 id="RateLimitFilter-InstanceStats"><a href="#RateLimitFilter-InstanceStats" class="headerlink" title="RateLimitFilter::InstanceStats"></a>RateLimitFilter::InstanceStats</h4><figure class="highlight c++"><table><tbody><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment"> * Struct definition for all tcp rate limit stats. @see stats_macros.h</span></span><br><span class="line"><span class="comment"> */</span></span><br><span class="line"><span class="class"><span class="keyword">struct</span> <span class="title">InstanceStats</span> {</span></span><br><span class="line">  ALL_TCP_RATE_LIMIT_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT)</span><br><span class="line">};</span><br></pre></td></tr></tbody></table></figure><p>有点懵逼，这货是一个结构体。。。还用了几个莫名其妙的宏。。。先看 <code>ALL_TCP_RATE_LIMIT_STATS</code> 写的啥：</p><figure class="highlight c++"><table><tbody><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment"> * All tcp rate limit stats. @see stats_macros.h</span></span><br><span class="line"><span class="comment"> */</span></span><br><span class="line"><span class="comment">// clang-format off</span></span><br><span class="line"><span class="meta">#<span class="meta-keyword">define</span> ALL_TCP_RATE_LIMIT_STATS(COUNTER, GAUGE)                                         \</span></span><br><span class="line">  COUNTER(total)                                                                         \</span><br><span class="line">  COUNTER(error)                                                                         \</span><br><span class="line">  COUNTER(over_limit)                                                                    \</span><br><span class="line">  COUNTER(ok)                                                                            \</span><br><span class="line">  COUNTER(failure_mode_allowed)                                                          \</span><br><span class="line">  COUNTER(cx_closed)                                                                     \</span><br><span class="line">  GAUGE  (active)</span><br><span class="line"><span class="comment">// clang-format on</span></span><br></pre></td></tr></tbody></table></figure><p>丫的是个嵌套宏，，，大意是执行传入参数 COUNTER, GAUGE 的宏。注释提到了 <code>stats_macros.h</code> ，看下：</p><figure class="highlight c++"><table><tbody><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#<span class="meta-keyword">pragma</span> once</span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="meta-keyword">include</span> <span class="meta-string">&lt;string&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="meta-keyword">include</span> <span class="meta-string">"envoy/stats/histogram.h"</span></span></span><br><span class="line"><span class="meta">#<span class="meta-keyword">include</span> <span class="meta-string">"envoy/stats/stats.h"</span></span></span><br><span class="line"></span><br><span class="line"><span class="keyword">namespace</span> Envoy {</span><br><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment"> * These are helper macros for allocating "fixed" stats throughout the code base in a way that</span></span><br><span class="line"><span class="comment"> * is also easy to mock and test. The general flow looks like this:</span></span><br><span class="line"><span class="comment"> *</span></span><br><span class="line"><span class="comment"> * Define a block of stats like this:</span></span><br><span class="line"><span class="comment"> *   #define MY_COOL_STATS(COUNTER, GAUGE, HISTOGRAM) \</span></span><br><span class="line"><span class="comment"> *     COUNTER(counter1)</span></span><br><span class="line"><span class="comment"> *     GAUGE(gauge1)</span></span><br><span class="line"><span class="comment"> *     HISTOGRAM(histogram1)</span></span><br><span class="line"><span class="comment"> *     ...</span></span><br><span class="line"><span class="comment"> *</span></span><br><span class="line"><span class="comment"> * Now actually put these stats somewhere, usually as a member of a struct:</span></span><br><span class="line"><span class="comment"> *   struct MyCoolStats {</span></span><br><span class="line"><span class="comment"> *     MY_COOL_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT, GENERATE_HISTOGRAM_STRUCT)</span></span><br><span class="line"><span class="comment"> *   };</span></span><br><span class="line"><span class="comment"> *</span></span><br><span class="line"><span class="comment"> * Finally, when you want to actually instantiate the above struct using a Stats::Pool, you do:</span></span><br><span class="line"><span class="comment"> *   MyCoolStats stats{</span></span><br><span class="line"><span class="comment"> *     MY_COOL_STATS(POOL_COUNTER(...), POOL_GAUGE(...), POOL_HISTOGRAM(...))};</span></span><br><span class="line"><span class="comment"> */</span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="meta-keyword">define</span> GENERATE_COUNTER_STRUCT(NAME) Stats::Counter&amp; NAME##_;</span></span><br><span class="line"><span class="meta">#<span class="meta-keyword">define</span> GENERATE_GAUGE_STRUCT(NAME) Stats::Gauge&amp; NAME##_;</span></span><br><span class="line"><span class="meta">#<span class="meta-keyword">define</span> GENERATE_HISTOGRAM_STRUCT(NAME) Stats::Histogram&amp; NAME##_;</span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="meta-keyword">define</span> FINISH_STAT_DECL_(X) + std::string(#X)),</span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="meta-keyword">define</span> POOL_COUNTER_PREFIX(POOL, PREFIX) (POOL).counter(PREFIX FINISH_STAT_DECL_</span></span><br><span class="line"><span class="meta">#<span class="meta-keyword">define</span> POOL_GAUGE_PREFIX(POOL, PREFIX) (POOL).gauge(PREFIX FINISH_STAT_DECL_</span></span><br><span class="line"><span class="meta">#<span class="meta-keyword">define</span> POOL_HISTOGRAM_PREFIX(POOL, PREFIX) (POOL).histogram(PREFIX FINISH_STAT_DECL_</span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="meta-keyword">define</span> POOL_COUNTER(POOL) POOL_COUNTER_PREFIX(POOL, <span class="meta-string">""</span>)</span></span><br><span class="line"><span class="meta">#<span class="meta-keyword">define</span> POOL_GAUGE(POOL) POOL_GAUGE_PREFIX(POOL, <span class="meta-string">""</span>)</span></span><br><span class="line"><span class="meta">#<span class="meta-keyword">define</span> POOL_HISTOGRAM(POOL) POOL_HISTOGRAM_PREFIX(POOL, <span class="meta-string">""</span>)</span></span><br><span class="line">} <span class="comment">// namespace Envoy</span></span><br></pre></td></tr></tbody></table></figure><p>大体意思看注释就知道了，这个宏主要是方便生成一个统计的数据结构。那将 <code>InstanceStats</code> 理解为一个数据结构即可，它的成员有 <code>Stats::Counter</code> 和 <code>Stats::Gauge</code> 。</p><h4 id="Stats-Counter"><a href="#Stats-Counter" class="headerlink" title="Stats::Counter"></a>Stats::Counter</h4><p>以 Counter 为例看下源码：</p><figure class="highlight c++"><table><tbody><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment"> * An always incrementing counter with latching capability. Each increment is added both to a</span></span><br><span class="line"><span class="comment"> * global counter as well as periodic counter. Calling latch() returns the periodic counter and</span></span><br><span class="line"><span class="comment"> * clears it.</span></span><br><span class="line"><span class="comment"> */</span></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Counter</span> :</span> <span class="keyword">public</span> <span class="keyword">virtual</span> Metric {</span><br><span class="line"><span class="keyword">public</span>:</span><br><span class="line">  <span class="keyword">virtual</span> ~Counter() {}</span><br><span class="line">  <span class="function"><span class="keyword">virtual</span> <span class="keyword">void</span> <span class="title">add</span><span class="params">(<span class="keyword">uint64_t</span> amount)</span> PURE</span>;</span><br><span class="line">  <span class="function"><span class="keyword">virtual</span> <span class="keyword">void</span> <span class="title">inc</span><span class="params">()</span> PURE</span>;</span><br><span class="line">  <span class="function"><span class="keyword">virtual</span> uint64_t <span class="title">latch</span><span class="params">()</span> PURE</span>;</span><br><span class="line">  <span class="function"><span class="keyword">virtual</span> <span class="keyword">void</span> <span class="title">reset</span><span class="params">()</span> PURE</span>;</span><br><span class="line">  <span class="function"><span class="keyword">virtual</span> uint64_t <span class="title">value</span><span class="params">()</span> <span class="keyword">const</span> PURE</span>;</span><br><span class="line">};</span><br></pre></td></tr></tbody></table></figure><p>这样一看，执行流程就明白了。回过头来 <code>InstanceStat</code> 是如何被初始化的呢？这个 Counter 的具体实现是什么？</p><h4 id="InstanceStat-初始化"><a href="#InstanceStat-初始化" class="headerlink" title="InstanceStat 初始化"></a>InstanceStat 初始化</h4><p><code>envoy/source/extensions/filters/network/ratelimit/ratelimit.cc</code> 下有一段代码生成 <code>InstanceStat</code>：</p><figure class="highlight c++"><table><tbody><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">InstanceStats Config::generateStats(<span class="keyword">const</span> <span class="built_in">std</span>::<span class="built_in">string</span>&amp; name, Stats::Scope&amp; scope) {</span><br><span class="line">  <span class="built_in">std</span>::<span class="built_in">string</span> final_prefix = fmt::format(<span class="string">"ratelimit.{}."</span>, name);</span><br><span class="line">  <span class="keyword">return</span> {ALL_TCP_RATE_LIMIT_STATS(POOL_COUNTER_PREFIX(scope, final_prefix),</span><br><span class="line">                                   POOL_GAUGE_PREFIX(scope, final_prefix))};</span><br><span class="line">}</span><br></pre></td></tr></tbody></table></figure><p>结合前面 <code>stats_macros.h</code> 的源码来看，实际调用了 <code>scope.counter()</code> 方法。看下 scope 代码。</p><h4 id="Stats-Scope"><a href="#Stats-Scope" class="headerlink" title="Stats::Scope"></a>Stats::Scope</h4><figure class="highlight c++"><table><tbody><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#<span class="meta-keyword">pragma</span> once</span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="meta-keyword">include</span> <span class="meta-string">&lt;cstdint&gt;</span></span></span><br><span class="line"><span class="meta">#<span class="meta-keyword">include</span> <span class="meta-string">&lt;memory&gt;</span></span></span><br><span class="line"><span class="meta">#<span class="meta-keyword">include</span> <span class="meta-string">&lt;string&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="meta-keyword">include</span> <span class="meta-string">"envoy/common/pure.h"</span></span></span><br><span class="line"><span class="meta">#<span class="meta-keyword">include</span> <span class="meta-string">"envoy/stats/histogram.h"</span></span></span><br><span class="line"><span class="meta">#<span class="meta-keyword">include</span> <span class="meta-string">"envoy/stats/stats_options.h"</span></span></span><br><span class="line"></span><br><span class="line"><span class="keyword">namespace</span> Envoy {</span><br><span class="line"><span class="keyword">namespace</span> Stats {</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Counter</span>;</span></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Gauge</span>;</span></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Histogram</span>;</span></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Scope</span>;</span></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">StatsOptions</span>;</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">typedef</span> <span class="built_in">std</span>::<span class="built_in">unique_ptr</span>&lt;Scope&gt; ScopePtr;</span><br><span class="line"><span class="keyword">typedef</span> <span class="built_in">std</span>::<span class="built_in">shared_ptr</span>&lt;Scope&gt; ScopeSharedPtr;</span><br><span class="line"></span><br><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment"> * A named scope for stats. Scopes are a grouping of stats that can be acted on as a unit if needed</span></span><br><span class="line"><span class="comment"> * (for example to free/delete all of them).</span></span><br><span class="line"><span class="comment"> */</span></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Scope</span> {</span></span><br><span class="line"><span class="keyword">public</span>:</span><br><span class="line">  <span class="keyword">virtual</span> ~Scope() {}</span><br><span class="line"></span><br><span class="line">  <span class="comment">/**</span></span><br><span class="line"><span class="comment">   * Allocate a new scope. <span class="doctag">NOTE:</span> The implementation should correctly handle overlapping scopes</span></span><br><span class="line"><span class="comment">   * that point to the same reference counted backing stats. This allows a new scope to be</span></span><br><span class="line"><span class="comment">   * gracefully swapped in while an old scope with the same name is being destroyed.</span></span><br><span class="line"><span class="comment">   * @param name supplies the scope's namespace prefix.</span></span><br><span class="line"><span class="comment">   */</span></span><br><span class="line">  <span class="function"><span class="keyword">virtual</span> ScopePtr <span class="title">createScope</span><span class="params">(<span class="keyword">const</span> <span class="built_in">std</span>::<span class="built_in">string</span>&amp; name)</span> PURE</span>;</span><br><span class="line"></span><br><span class="line">  <span class="comment">/**</span></span><br><span class="line"><span class="comment">   * Deliver an individual histogram value to all registered sinks.</span></span><br><span class="line"><span class="comment">   */</span></span><br><span class="line">  <span class="function"><span class="keyword">virtual</span> <span class="keyword">void</span> <span class="title">deliverHistogramToSinks</span><span class="params">(<span class="keyword">const</span> Histogram&amp; histogram, <span class="keyword">uint64_t</span> value)</span> PURE</span>;</span><br><span class="line"></span><br><span class="line">  <span class="comment">/**</span></span><br><span class="line"><span class="comment">   * @return a counter within the scope's namespace.</span></span><br><span class="line"><span class="comment">   */</span></span><br><span class="line">  <span class="function"><span class="keyword">virtual</span> Counter&amp; <span class="title">counter</span><span class="params">(<span class="keyword">const</span> <span class="built_in">std</span>::<span class="built_in">string</span>&amp; name)</span> PURE</span>;</span><br><span class="line"></span><br><span class="line">  <span class="comment">/**</span></span><br><span class="line"><span class="comment">   * @return a gauge within the scope's namespace.</span></span><br><span class="line"><span class="comment">   */</span></span><br><span class="line">  <span class="function"><span class="keyword">virtual</span> Gauge&amp; <span class="title">gauge</span><span class="params">(<span class="keyword">const</span> <span class="built_in">std</span>::<span class="built_in">string</span>&amp; name)</span> PURE</span>;</span><br><span class="line"></span><br><span class="line">  <span class="comment">/**</span></span><br><span class="line"><span class="comment">   * @return a histogram within the scope's namespace with a particular value type.</span></span><br><span class="line"><span class="comment">   */</span></span><br><span class="line">  <span class="function"><span class="keyword">virtual</span> Histogram&amp; <span class="title">histogram</span><span class="params">(<span class="keyword">const</span> <span class="built_in">std</span>::<span class="built_in">string</span>&amp; name)</span> PURE</span>;</span><br><span class="line"></span><br><span class="line">  <span class="comment">/**</span></span><br><span class="line"><span class="comment">   * @return a reference to the top-level StatsOptions struct, containing information about the</span></span><br><span class="line"><span class="comment">   * maximum allowable object name length and stat suffix length.</span></span><br><span class="line"><span class="comment">   */</span></span><br><span class="line">  <span class="keyword">virtual</span> <span class="keyword">const</span> Stats::<span class="function">StatsOptions&amp; <span class="title">statsOptions</span><span class="params">()</span> <span class="keyword">const</span> PURE</span>;</span><br><span class="line">};</span><br><span class="line"></span><br><span class="line">} <span class="comment">// namespace Stats</span></span><br><span class="line">} <span class="comment">// namespace Envoy</span></span><br></pre></td></tr></tbody></table></figure><p>可看到获取的是 scope 下的一个 counter，具体实现的 scope 是 <code>Stats::IsolatedScopeImpl</code> 。</p><h4 id="Stats-IsolatedScopeImpl"><a href="#Stats-IsolatedScopeImpl" class="headerlink" title="Stats::IsolatedScopeImpl"></a>Stats::IsolatedScopeImpl</h4><figure class="highlight c++"><table><tbody><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre></td><td class="code"><pre><span class="line"><span class="class"><span class="keyword">struct</span> <span class="title">IsolatedScopeImpl</span> :</span> <span class="keyword">public</span> Scope {</span><br><span class="line">  IsolatedScopeImpl(IsolatedStoreImpl&amp; parent, <span class="keyword">const</span> <span class="built_in">std</span>::<span class="built_in">string</span>&amp; prefix)</span><br><span class="line">      : parent_(parent), prefix_(Utility::sanitizeStatsName(prefix)) {}</span><br><span class="line"></span><br><span class="line">  <span class="comment">// Stats::Scope</span></span><br><span class="line">  <span class="function">ScopePtr <span class="title">createScope</span><span class="params">(<span class="keyword">const</span> <span class="built_in">std</span>::<span class="built_in">string</span>&amp; name)</span> override </span>{</span><br><span class="line">    <span class="keyword">return</span> ScopePtr{<span class="keyword">new</span> IsolatedScopeImpl(parent_, prefix_ + name)};</span><br><span class="line">  }</span><br><span class="line">  <span class="function"><span class="keyword">void</span> <span class="title">deliverHistogramToSinks</span><span class="params">(<span class="keyword">const</span> Histogram&amp;, <span class="keyword">uint64_t</span>)</span> override </span>{}</span><br><span class="line">  <span class="function">Counter&amp; <span class="title">counter</span><span class="params">(<span class="keyword">const</span> <span class="built_in">std</span>::<span class="built_in">string</span>&amp; name)</span> override </span>{ <span class="keyword">return</span> parent_.counter(prefix_ + name); }</span><br><span class="line">  <span class="function">Gauge&amp; <span class="title">gauge</span><span class="params">(<span class="keyword">const</span> <span class="built_in">std</span>::<span class="built_in">string</span>&amp; name)</span> override </span>{ <span class="keyword">return</span> parent_.gauge(prefix_ + name); }</span><br><span class="line">  <span class="function">Histogram&amp; <span class="title">histogram</span><span class="params">(<span class="keyword">const</span> <span class="built_in">std</span>::<span class="built_in">string</span>&amp; name)</span> override </span>{</span><br><span class="line">    <span class="keyword">return</span> parent_.histogram(prefix_ + name);</span><br><span class="line">  }</span><br><span class="line">  <span class="keyword">const</span> Stats::<span class="function">StatsOptions&amp; <span class="title">statsOptions</span><span class="params">()</span> <span class="keyword">const</span> override </span>{ <span class="keyword">return</span> parent_.statsOptions(); }</span><br><span class="line"></span><br><span class="line">  IsolatedStoreImpl&amp; parent_;</span><br><span class="line">  <span class="keyword">const</span> <span class="built_in">std</span>::<span class="built_in">string</span> prefix_;</span><br><span class="line">};</span><br></pre></td></tr></tbody></table></figure><p>可看到实际用了 <code>IsolatedStoreImpl</code> 的 <code>counter(name)</code> 方法。</p><h4 id="Stats-IsolatedStoreImpl"><a href="#Stats-IsolatedStoreImpl" class="headerlink" title="Stats::IsolatedStoreImpl"></a>Stats::IsolatedStoreImpl</h4><p><code>IsolatedStoreImpl</code> 是 <code>Stats::Store</code> 的一个子类，结合前文的概念，就知道这里终于走到了 store 层。</p><figure class="highlight c++"><table><tbody><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br></pre></td><td class="code"><pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">IsolatedStoreImpl</span> :</span> <span class="keyword">public</span> Store {</span><br><span class="line"><span class="keyword">public</span>:</span><br><span class="line">  IsolatedStoreImpl();</span><br><span class="line"></span><br><span class="line">  <span class="comment">// Stats::Scope</span></span><br><span class="line">  <span class="function">Counter&amp; <span class="title">counter</span><span class="params">(<span class="keyword">const</span> <span class="built_in">std</span>::<span class="built_in">string</span>&amp; name)</span> override </span>{ <span class="keyword">return</span> counters_.get(name); }</span><br><span class="line">  <span class="function">ScopePtr <span class="title">createScope</span><span class="params">(<span class="keyword">const</span> <span class="built_in">std</span>::<span class="built_in">string</span>&amp; name)</span> override</span>;</span><br><span class="line">  <span class="function"><span class="keyword">void</span> <span class="title">deliverHistogramToSinks</span><span class="params">(<span class="keyword">const</span> Histogram&amp;, <span class="keyword">uint64_t</span>)</span> override </span>{}</span><br><span class="line">  <span class="function">Gauge&amp; <span class="title">gauge</span><span class="params">(<span class="keyword">const</span> <span class="built_in">std</span>::<span class="built_in">string</span>&amp; name)</span> override </span>{ <span class="keyword">return</span> gauges_.get(name); }</span><br><span class="line">  <span class="function">Histogram&amp; <span class="title">histogram</span><span class="params">(<span class="keyword">const</span> <span class="built_in">std</span>::<span class="built_in">string</span>&amp; name)</span> override </span>{</span><br><span class="line">    Histogram&amp; histogram = histograms_.get(name);</span><br><span class="line">    <span class="keyword">return</span> histogram;</span><br><span class="line">  }</span><br><span class="line">  <span class="keyword">const</span> Stats::<span class="function">StatsOptions&amp; <span class="title">statsOptions</span><span class="params">()</span> <span class="keyword">const</span> override </span>{ <span class="keyword">return</span> stats_options_; }</span><br><span class="line"></span><br><span class="line">  <span class="comment">// Stats::Store</span></span><br><span class="line">  <span class="built_in">std</span>::<span class="built_in">vector</span>&lt;CounterSharedPtr&gt; counters() <span class="keyword">const</span> override { <span class="keyword">return</span> counters_.toVector(); }</span><br><span class="line">  <span class="built_in">std</span>::<span class="built_in">vector</span>&lt;GaugeSharedPtr&gt; gauges() <span class="keyword">const</span> override { <span class="keyword">return</span> gauges_.toVector(); }</span><br><span class="line">  <span class="built_in">std</span>::<span class="built_in">vector</span>&lt;ParentHistogramSharedPtr&gt; histograms() <span class="keyword">const</span> override {</span><br><span class="line">    <span class="keyword">return</span> <span class="built_in">std</span>::<span class="built_in">vector</span>&lt;ParentHistogramSharedPtr&gt;{};</span><br><span class="line">  }</span><br><span class="line"></span><br><span class="line"><span class="keyword">private</span>:</span><br><span class="line">  HeapStatDataAllocator alloc_;</span><br><span class="line">  IsolatedStatsCache&lt;Counter&gt; counters_;</span><br><span class="line">  IsolatedStatsCache&lt;Gauge&gt; gauges_;</span><br><span class="line">  IsolatedStatsCache&lt;Histogram&gt; histograms_;</span><br><span class="line">  <span class="keyword">const</span> StatsOptionsImpl stats_options_;</span><br><span class="line">};</span><br></pre></td></tr></tbody></table></figure><p><code>counter(name)</code> 方法其实就是从一个 vector 中获取对应 counter。</p><h2 id="Source"><a href="#Source" class="headerlink" title="Source"></a>Source</h2><p>先介绍下 <code>Stats::Source</code> ，通过这个类，可以访问缓存中的统计数据。</p><figure class="highlight c++"><table><tbody><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment"> * Provides cached access to a particular store's stats.</span></span><br><span class="line"><span class="comment"> */</span></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Source</span> {</span></span><br><span class="line"><span class="keyword">public</span>:</span><br><span class="line">  <span class="keyword">virtual</span> ~Source() {}</span><br><span class="line"></span><br><span class="line">  <span class="comment">/**</span></span><br><span class="line"><span class="comment">   * Returns all known counters. Will use cached values if already accessed and clearCache() hasn't</span></span><br><span class="line"><span class="comment">   * been called since.</span></span><br><span class="line"><span class="comment">   * @return std::vector&lt;CounterSharedPtr&gt;&amp; all known counters. Note: reference may not be valid</span></span><br><span class="line"><span class="comment">   * after clearCache() is called.</span></span><br><span class="line"><span class="comment">   */</span></span><br><span class="line">  <span class="keyword">virtual</span> <span class="keyword">const</span> <span class="built_in">std</span>::<span class="built_in">vector</span>&lt;CounterSharedPtr&gt;&amp; cachedCounters() PURE;</span><br><span class="line"></span><br><span class="line">  <span class="comment">/**</span></span><br><span class="line"><span class="comment">   * Returns all known gauges. Will use cached values if already accessed and clearCache() hasn't</span></span><br><span class="line"><span class="comment">   * been called since.</span></span><br><span class="line"><span class="comment">   * @return std::vector&lt;GaugeSharedPtr&gt;&amp; all known gauges. Note: reference may not be valid after</span></span><br><span class="line"><span class="comment">   * clearCache() is called.</span></span><br><span class="line"><span class="comment">   */</span></span><br><span class="line">  <span class="keyword">virtual</span> <span class="keyword">const</span> <span class="built_in">std</span>::<span class="built_in">vector</span>&lt;GaugeSharedPtr&gt;&amp; cachedGauges() PURE;</span><br><span class="line"></span><br><span class="line">  <span class="comment">/**</span></span><br><span class="line"><span class="comment">   * Returns all known parent histograms. Will use cached values if already accessed and</span></span><br><span class="line"><span class="comment">   * clearCache() hasn't been called since.</span></span><br><span class="line"><span class="comment">   * @return std::vector&lt;ParentHistogramSharedPtr&gt;&amp; all known histograms. Note: reference may not be</span></span><br><span class="line"><span class="comment">   * valid after clearCache() is called.</span></span><br><span class="line"><span class="comment">   */</span></span><br><span class="line">  <span class="keyword">virtual</span> <span class="keyword">const</span> <span class="built_in">std</span>::<span class="built_in">vector</span>&lt;ParentHistogramSharedPtr&gt;&amp; cachedHistograms() PURE;</span><br><span class="line"></span><br><span class="line">  <span class="comment">/**</span></span><br><span class="line"><span class="comment">   * Resets the cache so that any future calls to get cached metrics will refresh the set.</span></span><br><span class="line"><span class="comment">   */</span></span><br><span class="line">  <span class="function"><span class="keyword">virtual</span> <span class="keyword">void</span> <span class="title">clearCache</span><span class="params">()</span> PURE</span>;</span><br><span class="line">};</span><br></pre></td></tr></tbody></table></figure><h2 id="Sink"><a href="#Sink" class="headerlink" title="Sink"></a>Sink</h2><p>上面的流程追踪到 <code>sink-&gt;flush(source)</code> 方法，这里的 sink 实际是 <code>Stats::Sink</code> 的子类，具体的实现类在 <code>envoy/source/extensions/stat_sinks</code> 文件夹下，有</p><ul><li><a href="https://www.envoyproxy.io/docs/envoy/latest/api-v2/config/metrics/v2/stats.proto#envoy-api-msg-config-metrics-v2-statsdsink" target="_blank" rel="noopener">envoy.statsd</a> 支持 TCP/UDP</li><li><a href="https://www.envoyproxy.io/docs/envoy/latest/api-v2/config/metrics/v2/stats.proto#envoy-api-msg-config-metrics-v2-dogstatsdsink" target="_blank" rel="noopener">envoy.dog_statsd</a></li><li><a href="https://www.envoyproxy.io/docs/envoy/latest/api-v2/config/metrics/v2/metrics_service.proto#envoy-api-msg-config-metrics-v2-metricsserviceconfig" target="_blank" rel="noopener">envoy.metrics_service</a> 后端接一个符合某种标准的 grpc sink；</li><li><a href="https://www.envoyproxy.io/docs/envoy/latest/api-v2/config/metrics/v2/stats.proto#envoy-api-msg-config-metrics-v2-hystrixsink" target="_blank" rel="noopener">envoy.stat_sinks.hystrix</a> 暴露一个 url ，提供流式信息输出</li></ul><p>其中 statsd 和 dog_statsd 都是基于 statsd 格式，故使用 <code>stat_sinks/common/statsd</code> 下的共通代码。下面以 <code>envoy/source/extensions/stat_sinks/common/statsd/statsd.cc</code> 为例分析：</p><figure class="highlight c++"><table><tbody><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">//...</span></span><br><span class="line"><span class="keyword">void</span> UdpStatsdSink::flush(Stats::Source&amp; source) {</span><br><span class="line">  Writer&amp; writer = tls_-&gt;getTyped&lt;Writer&gt;();</span><br><span class="line">  <span class="keyword">for</span> (<span class="keyword">const</span> Stats::CounterSharedPtr&amp; counter : source.cachedCounters()) {</span><br><span class="line">    <span class="keyword">if</span> (counter-&gt;used()) {</span><br><span class="line">      <span class="keyword">uint64_t</span> delta = counter-&gt;latch();</span><br><span class="line">      writer.write(fmt::format(<span class="string">"{}.{}:{}|c{}"</span>, prefix_, getName(*counter), delta,</span><br><span class="line">                               buildTagStr(counter-&gt;tags())));</span><br><span class="line">    }</span><br><span class="line">  }</span><br><span class="line"></span><br><span class="line">  <span class="keyword">for</span> (<span class="keyword">const</span> Stats::GaugeSharedPtr&amp; gauge : source.cachedGauges()) {</span><br><span class="line">    <span class="keyword">if</span> (gauge-&gt;used()) {</span><br><span class="line">      writer.write(fmt::format(<span class="string">"{}.{}:{}|g{}"</span>, prefix_, getName(*gauge), gauge-&gt;value(),</span><br><span class="line">                               buildTagStr(gauge-&gt;tags())));</span><br><span class="line">    }</span><br><span class="line">  }</span><br><span class="line">}</span><br><span class="line"><span class="comment">//...</span></span><br></pre></td></tr></tbody></table></figure><p>如上是一个 Sink 子类的 flush 方法实现。可看到 statsd 是如何从 source 中取出所需数据的。</p><h1 id="总结"><a href="#总结" class="headerlink" title="总结"></a>总结</h1><p>回过头来看 envoy 为何这么设计，实际上与本文开头的设计初衷对应：</p><ul><li><strong>Linear throughput（线性吞吐）:</strong> At steady state, all stat allocations occur via the scope TLS cache. This requires no locks and scales very well to a high number of workers.</li><li><strong>Logically consistent during hot restart（热重启期间的逻辑一致）</strong>: Ultimately, all stats with the same name use the same backing storage in shared memory. This creates logical consistency between processes.</li><li><strong>Stats are contained within a scope, can be freed as a group, and can be overlapped（以scope为单位保存，以组为单位释放，可覆盖）</strong>: Scope 是具有相同前缀统计数据的逻辑集合。Scopes have completely independent central caches and TLS caches, along with independent per-stat reference counting. A scope can be removed and all of its stats will have their reference counts decremented and will potentially be freed.</li><li><strong>Adequate performance for dynamic stats</strong>: Lookups for dynamic stats happen via the scope TLS cache and use an O(1) hash table.</li></ul></div><footer class="post-footer"><div class="post-tags"> <a href="/tags/envoy/" rel="tag"># envoy</a> <a href="/tags/service-mesh/" rel="tag"># service mesh</a> <a href="/tags/源码分析/" rel="tag"># 源码分析</a></div><div class="post-nav"><div class="post-nav-next post-nav-item"><a href="/开发/ff55e9f.html" rel="next" title="Istio学习之Pilot-Discovery"><i class="fa fa-chevron-left"></i> Istio学习之Pilot-Discovery</a></div><span class="post-nav-divider"></span><div class="post-nav-prev post-nav-item"> <a href="/开发/d1717587.html" rel="prev" title="gRPC Bidirectional（双向流）源码分析">gRPC Bidirectional（双向流）源码分析<i class="fa fa-chevron-right"></i></a></div></div></footer></div></article></div></div><div class="comments" id="comments"></div></div><div class="sidebar-toggle"><div class="sidebar-toggle-line-wrap"><span class="sidebar-toggle-line sidebar-toggle-line-first"></span><span class="sidebar-toggle-line sidebar-toggle-line-middle"></span><span class="sidebar-toggle-line sidebar-toggle-line-last"></span></div></div><aside id="sidebar" class="sidebar"><div class="sidebar-inner"><ul class="sidebar-nav motion-element"><li class="sidebar-nav-toc sidebar-nav-active" data-target="post-toc-wrap"> 文章目录</li><li class="sidebar-nav-overview" data-target="site-overview-wrap"> 站点概览</li></ul><div class="site-overview-wrap sidebar-panel"><div class="site-overview"><div class="site-author motion-element" itemprop="author" itemscope="" itemtype="http://schema.org/Person"> <img class="site-author-image" itemprop="image" src="/images/blogAvatar.jpg" alt="moeyui"><p class="site-author-name" itemprop="name">moeyui</p><p class="site-description motion-element" itemprop="description">moeyui | moeyui's blog</p></div><nav class="site-state motion-element"><div class="site-state-item site-state-posts"> <a href="/archives/"><span class="site-state-item-count">39</span> <span class="site-state-item-name">日志</span></a></div><div class="site-state-item site-state-categories"> <a href="/categories/index.html"><span class="site-state-item-count">6</span> <span class="site-state-item-name">分类</span></a></div><div class="site-state-item site-state-tags"> <a href="/tags/index.html"><span class="site-state-item-count">36</span> <span class="site-state-item-name">标签</span></a></div></nav><div class="feed-link motion-element"><a href="/atom.xml" rel="alternate"><i class="fa fa-rss"></i> RSS</a></div><div class="links-of-author motion-element"><span class="links-of-author-item"><a href="https://github.com/moeyui1" title="GitHub → https://github.com/moeyui1" rel="noopener" target="_blank"><i class="fa fa-fw fa-github"></i> GitHub</a></span><span class="links-of-author-item"><a href="https://twitter.com/moeyui1" title="Twitter → https://twitter.com/moeyui1" rel="noopener" target="_blank"><i class="fa fa-fw fa-twitter"></i> Twitter</a></span></div></div></div><div class="post-toc-wrap motion-element sidebar-panel sidebar-panel-active"><div class="post-toc"><div class="post-toc-content"><ol class="nav"><li class="nav-item nav-level-1"><a class="nav-link" href="#设计初衷"><span class="nav-number">1.</span> <span class="nav-text">设计初衷</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#Overview"><span class="nav-number">2.</span> <span class="nav-text">Overview</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#结构"><span class="nav-number">3.</span> <span class="nav-text">结构</span></a><ol class="nav-child"><li class="nav-item nav-level-2"><a class="nav-link" href="#Store"><span class="nav-number">3.1.</span> <span class="nav-text">Store</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Stats"><span class="nav-number">3.2.</span> <span class="nav-text">Stats</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Flusher"><span class="nav-number">3.3.</span> <span class="nav-text">Flusher</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Sinks"><span class="nav-number">3.4.</span> <span class="nav-text">Sinks</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Admin"><span class="nav-number">3.5.</span> <span class="nav-text">Admin</span></a></li></ol></li><li class="nav-item nav-level-1"><a class="nav-link" href="#源码追踪"><span class="nav-number">4.</span> <span class="nav-text">源码追踪</span></a><ol class="nav-child"><li class="nav-item nav-level-2"><a class="nav-link" href="#执行流程"><span class="nav-number">4.1.</span> <span class="nav-text">执行流程</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#数据推送"><span class="nav-number">4.1.1.</span> <span class="nav-text">数据推送</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#埋点"><span class="nav-number">4.1.2.</span> <span class="nav-text">埋点</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#Network-FilterStatus-Filter-onNewConnection"><span class="nav-number">4.1.2.1.</span> <span class="nav-text">Network::FilterStatus Filter::onNewConnection()</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#RateLimitFilter-InstanceStats"><span class="nav-number">4.1.2.2.</span> <span class="nav-text">RateLimitFilter::InstanceStats</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Stats-Counter"><span class="nav-number">4.1.2.3.</span> <span class="nav-text">Stats::Counter</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#InstanceStat-初始化"><span class="nav-number">4.1.2.4.</span> <span class="nav-text">InstanceStat 初始化</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Stats-Scope"><span class="nav-number">4.1.2.5.</span> <span class="nav-text">Stats::Scope</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Stats-IsolatedScopeImpl"><span class="nav-number">4.1.2.6.</span> <span class="nav-text">Stats::IsolatedScopeImpl</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Stats-IsolatedStoreImpl"><span class="nav-number">4.1.2.7.</span> <span class="nav-text">Stats::IsolatedStoreImpl</span></a></li></ol></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Source"><span class="nav-number">4.2.</span> <span class="nav-text">Source</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Sink"><span class="nav-number">4.3.</span> <span class="nav-text">Sink</span></a></li></ol></li><li class="nav-item nav-level-1"><a class="nav-link" href="#总结"><span class="nav-number">5.</span> <span class="nav-text">总结</span></a></li></ol></div></div></div></div></aside></div></main><footer id="footer" class="footer"><div class="footer-inner"><div class="copyright">© 2015 – <span itemprop="copyrightYear">2020</span><span class="with-love" id="animate"><i class="fa fa-user"></i></span> <span class="author" itemprop="copyrightHolder">moeyui</span></div><div class="powered-by">由 <a href="https://hexo.io" class="theme-link" rel="noopener" target="_blank">Hexo</a> 强力驱动 v3.9.0</div> <span class="post-meta-divider">|</span><div class="theme-info">主题 – <a href="https://theme-next.org" class="theme-link" rel="noopener" target="_blank">NexT.Gemini</a> v6.7.0</div></div></footer><div class="back-to-top"><i class="fa fa-arrow-up"></i></div></div><script src="//cdn1.lncld.net/static/js/3.11.1/av-min.js"></script><script src="//unpkg.com/valine/dist/Valine.min.js"></script><style>.copy-btn{display:inline-block;padding:6px 12px;font-size:13px;font-weight:700;line-height:20px;color:#333;white-space:nowrap;vertical-align:middle;cursor:pointer;background-color:#eee;background-image:linear-gradient(#fcfcfc,#eee);border:1px solid #d5d5d5;border-radius:3px;user-select:none;outline:0}.highlight-wrap .copy-btn{transition:opacity .3s ease-in-out;opacity:0;padding:2px 6px;position:absolute;right:4px;top:8px}.highlight-wrap .copy-btn:focus,.highlight-wrap:hover .copy-btn{opacity:1}.highlight-wrap{position:relative}</style><script src="/bundle.js"></script><script>"[object Function]"!==Object.prototype.toString.call(window.Promise)&&(window.Promise=null);var GUEST=["nick","mail","link"],guest="nick,mail,link";guest=guest.split(",").filter(function(e){return-1<GUEST.indexOf(e)}),new Valine({el:"#comments",verify:!1,notify:!1,appId:"Nk9tNzbllEl79DVtRiOuLmbm-gzGzoHsz",appKey:"uPzxSfSzF95Gav6vGx2dOhCq",placeholder:"Just go go",avatar:"retro",meta:guest,pageSize:"10",visitor:!0}),$(".highlight").each(function(e,t){var n=$("<div>").addClass("highlight-wrap");$(t).after(n),n.append($("<button>").addClass("copy-btn").append("复制").on("click",function(e){var t=$(this).parent().find(".code").find(".line").map(function(e,t){return $(t).text()}).toArray().join("\n"),n=document.createElement("textarea"),o=document.createRange(),i=window.getSelection(),a=window.pageYOffset||document.documentElement.scrollTop;n.style.top=a+"px",n.style.position="absolute",n.style.opacity="0",n.value=t,n.textContent=t,n.contentEditable=!0,n.readOnly=!1,document.body.appendChild(n),o.selectNode(n),i.removeAllRanges(),i.addRange(o),n.setSelectionRange(0,t.length);document.execCommand("copy");n.blur(),$(this).blur()})).on("mouseleave",function(e){var t=$(this).find(".copy-btn");setTimeout(function(){t.text("复制")},300)}).append(t)})</script></body></html>